{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Handwritten Digit classification Using Neural Networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " First we import tensorflow and load the dataset from tensorflow.examples.tutorial.mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/mnist/train-images-idx3-ubyte.gz\n",
      "Extracting data/mnist/train-labels-idx1-ubyte.gz\n",
      "Extracting data/mnist/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/mnist/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"data/mnist/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    " Now, we will see what we got in our data,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No of images in training set (55000, 784)\n",
      "No of labels in training set (55000, 10)\n",
      "No of images in test set (10000, 784)\n",
      "No of labels in test set (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "print(\"No of images in training set {}\".format(mnist.train.images.shape))\n",
    "print(\"No of labels in training set {}\".format(mnist.train.labels.shape))\n",
    "\n",
    "print(\"No of images in test set {}\".format(mnist.test.images.shape))\n",
    "print(\"No of labels in test set {}\".format(mnist.test.labels.shape))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "We have 55000 images in the training set and each image is of size 784 and we have 10\n",
    "labels which are actually 0 to 9. Similarly, we have 10000 images in the test set. \n",
    "\n",
    "Now we plot an input image to see how it looks like, "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f7cdcdba990>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADphJREFUeJzt3X+sVPWZx/HPs3cpAWkihssPLUht\nzLqEKIWBEF2Na2PFTQ02oQbUypqm8EeNNiFmlWjqP2t0Y9tF3TS5IAESSktsWRB1F4IrLnElDmqQ\nLipYry0LgYugpX8gAZ794x6aK975nmHmzJyB5/1KzMyc55w5TwY/98zMd875mrsLQDx/VXYDAMpB\n+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBPXX7dzZqFGjfOLEie3cJRBKb2+vDh8+bPWs21T4\nzWyWpCWSuiQtc/cnUutPnDhR1Wq1mV0CSKhUKnWv2/DbfjPrkvRvkm6VNEnSPDOb1OjzAWivZj7z\nz5C0191/7+4nJP1K0uxi2gLQas2E/zJJfxzweF+27AvMbIGZVc2s2tfX18TuABSpmfAP9qXCl84P\ndvced6+4e6W7u7uJ3QEoUjPh3ydp/IDHX5O0v7l2ALRLM+F/U9KVZvZ1M/uKpLmSNhTTFoBWa3io\nz91Pmtl9kv5T/UN9y939d4V1BqClmhrnd/eXJL1UUC8A2oif9wJBEX4gKMIPBEX4gaAIPxAU4QeC\nIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEH\ngiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxBUU7P0mlmvpGOSTkk66e6VIpqK5tSpU8l6b29vw8/9\n0UcfJeubN29O1pctW5asHz169Jx7OuOee+5J1q+55ppk/d57703WL7744nPuKZKmwp/5e3c/XMDz\nAGgj3vYDQTUbfpe0ycx2mNmCIhoC0B7Nvu2/zt33m9loSZvN7D13f23gCtkfhQWSNGHChCZ3B6Ao\nTR353X1/dntI0jpJMwZZp8fdK+5e6e7ubmZ3AArUcPjN7CIz++qZ+5K+LWlXUY0BaK1m3vaPkbTO\nzM48zy/d/T8K6QpAy5m7t21nlUrFq9Vq2/bXKY4cOZKsz5kzJ1nfunVrke18Qd6/f/bHvRR5vV11\n1VXJ+ttvv12zNnTo0IZ66nSVSkXVarWufzSG+oCgCD8QFOEHgiL8QFCEHwiK8ANBFXFWH3I8+eST\nyXorh/Ly5J0Wm3fa7YgRIxre9/vvv5+s33333U1tP3v27Jq1F198MbltV1dXsn4h4MgPBEX4gaAI\nPxAU4QeCIvxAUIQfCIrwA0Exzt8GCxcuTNafeeaZZP3zzz8vsp0v2LlzZ7I+ffr0ZH348OEN73va\ntGnJ+pAhQ5L1uXPnJuupy5KfOHEiue2wYcOS9QsBR34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIpx\n/ja44oorkvWNGzcm60uXLk3W165de849nXHXXXcl63lj7a00derU0vYdAUd+ICjCDwRF+IGgCD8Q\nFOEHgiL8QFCEHwgqd5zfzJZL+o6kQ+4+OVt2iaRfS5ooqVfSHe5+tHVtXthuuummpupr1qwpsp2O\ncfjw4WS9ndPLX4jqOfKvkDTrrGUPSdri7ldK2pI9BnAeyQ2/u78m6chZi2dLWpndXynp9oL7AtBi\njX7mH+PuByQpux1dXEsA2qHlX/iZ2QIzq5pZta+vr9W7A1CnRsN/0MzGSVJ2e6jWiu7e4+4Vd690\nd3c3uDsARWs0/Bskzc/uz5e0vph2ALRLbvjNbI2k/5H0N2a2z8x+IOkJSTeb2R5JN2ePAZxHcsf5\n3X1ejdK3Cu4FwZw8eTJZf+SRR5J1M0vWhw4des49RcIv/ICgCD8QFOEHgiL8QFCEHwiK8ANBcelu\nlKanpydZ37JlS7KeN9S3ePHimrUIU3Dn4cgPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Exzn+BO378\neLJ+7NixZH3v3r3J+uuvv56sv/HGGzVreVOT5xk9On3pyIcffrip57/QceQHgiL8QFCEHwiK8ANB\nEX4gKMIPBEX4gaAY5z8PfPbZZ8n6yy+/XLP2+OOPJ7fdtWtXsp53znwrXXrppcn6pk2bkvWurq4i\n27ngcOQHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaByx/nNbLmk70g65O6Ts2WPSfqhpL5stcXu/lKr\nmoxuz549yfqdd97Z8HO7e8PbttqUKVOS9UmTJrWpkwtTPUf+FZJmDbL85+4+JfuP4APnmdzwu/tr\nko60oRcAbdTMZ/77zGynmS03s5GFdQSgLRoN/y8kfUPSFEkHJP201opmtsDMqmZW7evrq7UagDZr\nKPzuftDdT7n7aUlLJc1IrNvj7hV3r3R3dzfaJ4CCNRR+Mxs34OF3JaVPDQPQceoZ6lsj6UZJo8xs\nn6SfSLrRzKZIckm9kha2sEcALZAbfnefN8ji51rQC2oYP358sj5nzpyatW3btiW3zRvnv/7665P1\nmTNnJuup6/a/8MILyW1fffXVZH3Hjh3J+rRp05L16PiFHxAU4QeCIvxAUIQfCIrwA0ERfiAoLt19\nHhgzZkyyvnbt2jZ1Uqz7778/WX/22WeT9QcffDBZf+WVV865p0g48gNBEX4gKMIPBEX4gaAIPxAU\n4QeCIvxAUGHG+U+fPp2sf/LJJ8l6aiz9+PHjyW0XLVqUrEd12223Jet54/zvvfdeke2Ew5EfCIrw\nA0ERfiAowg8ERfiBoAg/EBThB4IKM86fNxY/duzYhp97/fr1DW97oUu97k8//XQbO8HZOPIDQRF+\nICjCDwRF+IGgCD8QFOEHgiL8QFC54/xmNl7SKkljJZ2W1OPuS8zsEkm/ljRRUq+kO9z9aOtaba28\nqapTJk2aVGAnneXUqVPJek9PT7K+atWqmrXt27cnt837N8mbuhxp9Rz5T0pa5O5/K2mmpB+Z2SRJ\nD0na4u5XStqSPQZwnsgNv7sfcPe3svvHJO2WdJmk2ZJWZqutlHR7q5oEULxz+sxvZhMlfVPSdklj\n3P2A1P8HQtLoopsD0Dp1h9/MRkj6jaQfu/ufzmG7BWZWNbNqX19fIz0CaIG6wm9mQ9Qf/NXu/tts\n8UEzG5fVx0k6NNi27t7j7hV3r3R3dxfRM4AC5IbfzEzSc5J2u/vPBpQ2SJqf3Z8viVPbgPNIPaf0\nXifp+5LeNbN3smWLJT0haa2Z/UDSHyR9rzUtFqP/b1htI0eOTNY//fTTmrVZs2Ylt928eXOyfvnl\nlyfreVKXJc87lfn5559P1lesWJGsb926NVlPGT58eLJ+6623JusrV65M1pGWG3533yapVnK+VWw7\nANqFX/gBQRF+ICjCDwRF+IGgCD8QFOEHggpz6e5hw4Yl6x9//HGyPmHChJq1Dz/8MLntjBkzkvVH\nH300WZ85c2aynroE9urVq5Pb5p02m/f7iDzTp0+vWVu2bFly28mTJze1b6Rx5AeCIvxAUIQfCIrw\nA0ERfiAowg8ERfiBoMKM8+cZMWJEsr5x48aatVtuuSW5bd7lyx544IFkvUxTp05N1tetW5esjxkz\npmZtyJAhDfWEYnDkB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgGOev07XXXluz9sEHHyS3XbJkSbKe\nd1770aONz3x+9dVXJ+tPPfVUsn7DDTck64zVn7848gNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUFbH\nddvHS1olaayk05J63H2JmT0m6YeSzpysvtjdX0o9V6VS8Wq12nTTAAZXqVRUrVbrmmyhnh/5nJS0\nyN3fMrOvStphZpuz2s/dPf0rEQAdKTf87n5A0oHs/jEz2y3pslY3BqC1zukzv5lNlPRNSduzRfeZ\n2U4zW25mI2tss8DMqmZWzbucFYD2qTv8ZjZC0m8k/djd/yTpF5K+IWmK+t8Z/HSw7dy9x90r7l7p\n7u4uoGUARagr/GY2RP3BX+3uv5Ukdz/o7qfc/bSkpZLSs1EC6Ci54bf+aVqfk7Tb3X82YPm4Aat9\nV9Ku4tsD0Cr1fNt/naTvS3rXzN7Jli2WNM/MpkhySb2SFrakQwAtUc+3/dskDTZumBzTB9DZ+IUf\nEBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gqNxLdxe6M7M+\nSR8PWDRK0uG2NXBuOrW3Tu1LordGFdnb5e5e1/Xy2hr+L+3crOruldIaSOjU3jq1L4neGlVWb7zt\nB4Ii/EBQZYe/p+T9p3Rqb53al0RvjSqlt1I/8wMoT9lHfgAlKSX8ZjbLzN43s71m9lAZPdRiZr1m\n9q6ZvWNmpU4pnE2DdsjMdg1YdomZbTazPdntoNOkldTbY2b2f9lr946Z/UNJvY03s/8ys91m9jsz\neyBbXuprl+irlNet7W/7zaxL0geSbpa0T9Kbkua5+/+2tZEazKxXUsXdSx8TNrMbJP1Z0ip3n5wt\n+xdJR9z9iewP50h3/6cO6e0xSX8ue+bmbEKZcQNnlpZ0u6R/VImvXaKvO1TC61bGkX+GpL3u/nt3\nPyHpV5Jml9BHx3P31yQdOWvxbEkrs/sr1f8/T9vV6K0juPsBd38ru39M0pmZpUt97RJ9laKM8F8m\n6Y8DHu9TZ0357ZI2mdkOM1tQdjODGJNNm35m+vTRJfdzttyZm9vprJmlO+a1a2TG66KVEf7BZv/p\npCGH69x9qqRbJf0oe3uL+tQ1c3O7DDKzdEdodMbropUR/n2Sxg94/DVJ+0voY1Duvj+7PSRpnTpv\n9uGDZyZJzW4PldzPX3TSzM2DzSytDnjtOmnG6zLC/6akK83s62b2FUlzJW0ooY8vMbOLsi9iZGYX\nSfq2Om/24Q2S5mf350taX2IvX9ApMzfXmllaJb92nTbjdSk/8smGMv5VUpek5e7+z21vYhBmdoX6\nj/ZS/ySmvyyzNzNbI+lG9Z/1dVDSTyT9u6S1kiZI+oOk77l72794q9Hbjep/6/qXmZvPfMZuc29/\nJ+m/Jb0r6XS2eLH6P1+X9tol+pqnEl43fuEHBMUv/ICgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIP\nBPX/lUYsit1vv9AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7cdeb88250>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "img1 = mnist.train.images[41].reshape(28,28)\n",
    "plt.imshow(img1, cmap='Greys')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we define hyperparameters which is learning rate, batch size and number of epcohs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.1\n",
    "batch_size = 100\n",
    "epochs = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "Let us start building our network. We build the 2 layer neural network with one input\n",
    "layer and one hidden layer and one output layer which predicts handwritten digit.\n",
    "\n",
    "First, we define the placeholders for our input and output. As our input data shape is 784,\n",
    "we can define input placeholder as"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "What does the None implies? None specifies the number of samples (batch size) passed\n",
    "which will be decided dynamically at run time.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since we have 10 classes as output, we can define output placeholder as,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    " Now, we define the weight and bias between an input to the hidden layer as w_xh and\n",
    "b_h respectively. We initialize the weight matrix with values randomly drawing from a\n",
    "normal distribution with standard deviation of 0.03."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w_ih = tf.Variable(tf.random_normal([784, 300], stddev=0.03), name='w_ih')\n",
    "b_h = tf.Variable(tf.random_normal([300]), name='b_h')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Next, we define the weights and bias between our hidden layer to the output layer as w_hy\n",
    "and b_y respectively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w_hy = tf.Variable(tf.random_normal([300, 10], stddev=0.03), name='w_hy')\n",
    "b_y = tf.Variable(tf.random_normal([10]), name='b_y')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us perform the forward propagation now, recall the steps we peformed in forward\n",
    "propagation. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "z1 = tf.add(tf.matmul(x, w_ih), b_h)\n",
    "a1 = tf.nn.relu(z1)\n",
    "z2 = tf.add(tf.matmul(a1, w_hy), b_y)\n",
    "yhat = tf.nn.softmax(z2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " We define our cost function cross entropy loss as,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cross_entropy = tf.reduce_mean(-tf.reduce_sum(y * tf.log(yhat), reduction_indices=[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Our objective is to minimize the cost function. We can minimize the cost function by\n",
    "propagating the network backward and perform gradient descent. With tensorflow we\n",
    "don't have to manually calculate the gradients, we can use tensorflow's built in gradient\n",
    "descent optimizer function as,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "optimiser = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To evaluate our model, we will calculate accuracy as follows,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(yhat, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    " As we know that tensorflow runs by building the computation graph, whatever we have\n",
    "written so far, will actually run only if we start the tensorflow session. So let's do that.\n",
    "\n",
    "First, initialize tensorflow variables,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init_op = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    " Now start the tensorflow session and start training the model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Epoch:', 1, 'cost =0.541')\n",
      "('Epoch:', 2, 'cost =0.292')\n",
      "('Epoch:', 3, 'cost =0.238')\n",
      "('Epoch:', 4, 'cost =0.196')\n",
      "('Epoch:', 5, 'cost =0.165')\n",
      "('Epoch:', 6, 'cost =0.142')\n",
      "('Epoch:', 7, 'cost =0.124')\n",
      "('Epoch:', 8, 'cost =0.111')\n",
      "('Epoch:', 9, 'cost =0.100')\n",
      "('Epoch:', 10, 'cost =0.090')\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "   sess.run(init_op)\n",
    "   total_batch = int(len(mnist.train.labels) / batch_size)\n",
    "    \n",
    "   for epoch in range(epochs):\n",
    "        avg_cost = 0\n",
    "        \n",
    "        for i in range(total_batch):\n",
    "            batch_x, batch_y = mnist.train.next_batch(batch_size=batch_size)\n",
    "            \n",
    "            _, c = sess.run([optimiser, cross_entropy], \n",
    "                         feed_dict={x: batch_x, y: batch_y})\n",
    "            \n",
    "            avg_cost += c / total_batch\n",
    "            \n",
    "        print(\"Epoch:\", (epoch + 1), \"cost =\"\"{:.3f}\".format(avg_cost))\n",
    "        \n",
    "   print(sess.run(accuracy, feed_dict={x: mnist.test.images, y: mnist.test.labels}))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
